import requests
from requests.exceptions import ConnectionError

base_headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7'
}

def get_page(url, options={}):
    """
    抓取代理
    :param url: 
    :param options:参数
    :return: 网页内容
    :type: get
    """
    headers = dict(base_headers, **options)
    print('正在抓取', url)
    try:
        response = requests.get(url, headers=headers)
        print('抓取成功', url, response.status_code)
        if response.status_code == 200:
            # utils.py中定义，根据RFC2612,如果HTTP头部content-typ中MIMR是text/*,且没有设置charset，假设编码方式为ISO-8859-1。
            if response.encoding == 'ISO-8859-1':
                encodings = requests.utils.get_encodings_from_content(response.text) # requests得到响应后，自动调用get_encoding_from_headers获取编码方式
                if encodings:
                    encoding = encodings[0]
                else:
                    encoding = requests.apparent_encoding
                encode_content = response.content.decode(encoding, 'replace')
                return encode_content
            return response.text
    except ConnectionError:
        print('抓取失败', url)
        return None